<?php

class spider
{

    function spider ($url)
    {
        $this->touched = array($url);
        $this->do_spider(array($url));
    }
    
    function do_spider($urls)
    {
        for ($i=0;$i<count($urls);$i++)
        {
            $recurls	    = array();
            $current        = $urls[$i];
            print 'current: '.$current."<br />\n";
            flush();
            if ($code       = file($current))
            {
                $links   = $this->parselinks(join("",$code),$current);
                for ($i=0;$i<count($links);$i++)
                {
                    $links[$i] = preg_replace('/#.*$/','',$links[$i]);
                    if (!in_array($links[$i],$this->touched)) 
                    {
                        $recurls[] = $links[$i];
                        if (count($this->touched)>100)
                        {
                            var_dump($this->touched);exit;
                        }
                        $this->touched[] = $links[$i]; 
                        print $links[$i].'<br>';
                    }
                }
                flush();
                //$this->index($code);
                unset($code);
                $this->do_spider($recurls);
            }
        }
    }
    
    function parselinks($code,$currenturl)
    {
        $purl = parse_url($currenturl);
        if ($purl['scheme']!='http') return;
        $parr = split("/",$purl['path']);
        array_pop($parr);
        $url  = 'http://'.$purl['host'];
        while (preg_match('/^.*?<(?:a|frame).*?(?:href|src)="(.*?)"(.*)$/is',$code,$matches))
        {
            $plink = parse_url($matches[1]);
            if (!$plink['scheme'])
            {
                if (preg_match('|((?:\.\./)+)(.*)$|',$plink['path'],$pathm))
                {
                    $absurl = $this->makeAbsUrl($url,$parr,$pathm);
                } 
                else
                {
                    $absurl = $url.join('/',$parr).'/'.$plink['path'];
                }
                if ($absurl)
                {
                    if ($plink['query'])
                        $absurl .= '?'.$plink['query'];
                    $urls[] = $absurl;
                }
            } 
            elseif (($plink['scheme']=='http') && ($plink['host']==$purl['host']))
            {
                $urls[] = $matches[1];
            }
            $code   = $matches[2];
        }
        return $urls;
    }
    
    function makeAbsUrl ($url,$parr,$pathm)
    {
        $backsteps = count(split('/',$pathm[1]))-1;
        $pcount    = count($parr);
        if ($pcount<$backsteps) return;
        $steps = $pcount-$backsteps;
        for ($i=0;$i<$steps;$i++)
        {
            $path .= $parr[$i].'/';
        }
        return $url.$path.$pathm[2];
    }

}

?>